package com.allbluefish.learn.mongo;

import static org.junit.Assert.assertTrue;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XPather;
import org.htmlcleaner.XPatherException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.document.mongodb.MongoTemplate;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

import com.allbluefish.learn.model.AncientBook;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations={"classpath*:**/*Context.xml"})
public class SaveToMongo {

	@Autowired
    private MongoTemplate mongoTemplate;
	
	private static final String DEFAULT_COLLECTIO_NNAME = "blue";
	
	private List<String> urls = Lists.newArrayList();
	
	private List<String> allUrls = Lists.newArrayList();
	
	File Lines50;
	File Lines5;
	File Lines1000;
	File Lines2000;
	File Lines3000;
	File Lines4000;

    @Before
    public void setUp() {
    	
    	ClassLoader classLoader = this.getClass().getClassLoader();
    	
		this.Lines50 = new File(classLoader.getResource("50.htm").getPath());
    	this.Lines5 = new File(classLoader.getResource("5.htm").getPath());
    	this.Lines1000 = new File(classLoader.getResource("1000.htm").getPath());
    	this.Lines2000 = new File(classLoader.getResource("2000.htm").getPath());
    	this.Lines3000 = new File(classLoader.getResource("3000.htm").getPath());
    	this.Lines4000 = new File(classLoader.getResource("4000.htm").getPath());
    	
        if (!this.mongoTemplate.getCollectionNames().contains(DEFAULT_COLLECTIO_NNAME)) {
            this.mongoTemplate.createCollection(DEFAULT_COLLECTIO_NNAME);
        }
        
        getURL();
    }

    @Test
    public void createCollection() {
        assertTrue(this.mongoTemplate.getCollectionNames().contains(DEFAULT_COLLECTIO_NNAME));
    }
    
    @Test
    public void saveBook(){
    	AncientBook ancientBook = new AncientBook();
    	ancientBook.setAppName("廖子南");
    	this.mongoTemplate.save(ancientBook);
    }
    
    private String getXpathExpression(String reg){
		return "//td[@style=\"" + reg + "\"]";
	}
	
    

	//著录编号
	private static final String reg_1 = "border-bottom:#e8e3d3 solid 1px; border-top:#d6d6d6 solid 1px;";
	private static final String reg_2 = "border-bottom:#e8e3d3 solid 1px;";
	private static final String reg_3 = "border-bottom:#e8e3d3 solid 1px;border-right:#d6d6d6 solid 1px;";
	private static final String reg_4 = "border-bottom:#e8e3d3 solid 1px; border-right:#d6d6d6 solid 1px;";
	private static final String reg_5 = "border-bottom:#d6d6d6 solid 1px; border-right:#d6d6d6 solid 1px;";
    
	private String replaceCharacter(String s){
		String[] cs = {"\r\n"," ","javascript:imageLargeShow('","',800,600);"};		
 		for(String c : cs){
 			s = s.replace(c, "");
 		}
 		return s.trim();
 	}
    
	private boolean getTextFromTag(String TagName, TagNode node) {
		String nodeText = replaceCharacter(node.getText().toString());
		return nodeText.contains(TagName);
	}
	
	private String getNextHtml(HtmlCleaner cleaner, Iterator<TagNode> i) {
		String html = cleaner.getInnerHtml(i.next());
		return replaceCharacter(html);
	}
	
	@Test
	@SuppressWarnings("unchecked")
	public void saveBookFromHtml_V2(){

		CleanerProperties props = new CleanerProperties();
		 
		// set some properties to non-default values
		props.setTranslateSpecialEntities(true);
		props.setTransResCharsToNCR(true);
		props.setOmitComments(true);
		
		HtmlCleaner cleaner = new HtmlCleaner(props);
		
		TagNode tagNode = null;
		int j = 0;
		for (String url : this.allUrls) {
			try {
				AncientBook book = new AncientBook();
				URL getUrl = new URL(url);
				HttpURLConnection connection = (HttpURLConnection) getUrl.openConnection();
				connection.setConnectTimeout(7000);
				connection.setReadTimeout(7000);
				connection.connect();
				/**
				 * 设置编码,否则中文乱码
				 */
				BufferedReader reader = new BufferedReader(
						new InputStreamReader(connection.getInputStream(), "utf-8"));
				
				tagNode = cleaner.clean(reader);
				List<TagNode> nodes = tagNode.getElementListByName("table", true);
				List<TagNode> nodes2 = nodes.get(6).getElementListByName("td", true);
				Iterator<TagNode> i = nodes2.iterator();
				while (i.hasNext()) {
					initBook(cleaner, book, i);
				}
				
				book.setDetailedLink(url);
				book.setSource("中华古籍善本国际联合书目系统");
				
				XPather xPather = new XPather("//a[//span[@class=\"dhzt\"]]");
				Object[] ns = xPather.evaluateAgainstNode(tagNode);
				
				for (Object o : ns) {
					String text = replaceCharacter(((TagNode) o).getText().toString());
					if(text.indexOf("下载图片") == 0){
						String link = ((TagNode) o).getAttributeByName("href");
						ImmutableList<String> of = ImmutableList.of("http://res4.nlc.gov.cn" + link);
						book.setBookImages(of);
					}
				}

				
				this.mongoTemplate.save(book);
				System.out.println(url);
				
			} catch (MalformedURLException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
				this.urls.add(url);
			} catch (XPatherException e) {
				e.printStackTrace();
			} catch (Exception e) {
				e.printStackTrace();
			}
			
			j = j + 1;
			System.out.println("保存到了第" + j + "条");
		}
	
		catchRemain();
	}

	private void initBook(HtmlCleaner cleaner, AncientBook book,
			Iterator<TagNode> i) {
		TagNode t = i.next();
		if(getTextFromTag("著录编号",t)){
			String content = getNextText(i);
			System.out.println("著录编号" + "@@@@@@@@@@@@" + content);
			book.setIndexNumber(content);
		}

		if(getTextFromTag("正题名",t)){
			String content = getNextText(i);
			System.out.println("正题名" + "@@@@@@@@@@@@" + content);
			book.setBookName(content);
		}

		if(getTextFromTag("其他题名",t)){
			String content = getNextHtml(cleaner,i);
			System.out.println("其他题名" + "@@@@@@@@@@@@" + content);
			book.setOtherBookNames(Arrays.asList(content.split("<br/>")));
		}
		
		if(getTextFromTag("版本类型",t)){
			String content = getNextText(i);
			System.out.println("版本类型" + "@@@@@@@@@@@@" + content);
			book.setVersion(content);
		}
		
		if(getTextFromTag("版本信息",t)){
			String content = getNextText(i);
			System.out.println("版本信息" + "@@@@@@@@@@@@" + content);
			String version = book.getVersion();
			book.setVersion(version != "" ? version + " " + content : "");
		}
		
		if(getTextFromTag("责任者",t)){
			String content = getNextText(i);
			System.out.println("责任者" + "@@@@@@@@@@@@" + content);
			book.setAuthors(Arrays.asList(content.split(";")));
		}
		
		if(getTextFromTag("载体形态",t)){
			String content = getNextText(i);
			System.out.println("载体形态" + "@@@@@@@@@@@@" + content);
			book.setCarrierShape(content);
		}
		
		if(getTextFromTag("行款版式",t)){
			String content = getNextText(i);
			System.out.println("行款版式" + "@@@@@@@@@@@@" + content);
			book.setFrame(content);
		}
		
		if(getTextFromTag("附注",t)){
			String content = getNextText(i);
			//content = content.replace("<br/>", " ");
			//content = content.replace("&#34;", "'");
			System.out.println("附注" + "@@@@@@@@@@@@" + content);
			book.setAppendix(content);
		}
		
		if(getTextFromTag("四部分类",t)){
			String content = getNextText(i);
			System.out.println("四部分类" + "@@@@@@@@@@@@" + content);
			book.setCategory(content);
		}
	}

	@SuppressWarnings("unchecked")
	private void catchRemain() {
		Iterator<String> u = this.urls.iterator();
		HtmlCleaner cleaner = new HtmlCleaner();
		TagNode tagNode = null;
		List<String> succs = Lists.newArrayList(); 
		String url = "";
		int j = 0;
		while (u.hasNext()) {
			try {
				AncientBook book = new AncientBook();
				url = u.next();
				URL getUrl = new URL(url);
				HttpURLConnection connection = (HttpURLConnection) getUrl.openConnection();
				connection.setConnectTimeout(7000);
				connection.setReadTimeout(7000);
				connection.connect();
				/**
				 * 设置编码,否则中文乱码
				 */
				BufferedReader reader = new BufferedReader(
						new InputStreamReader(connection.getInputStream(), "utf-8"));
				
				tagNode = cleaner.clean(reader);
				List<TagNode> nodes = tagNode.getElementListByName("table", true);
				List<TagNode> nodes2 = nodes.get(6).getElementListByName("td", true);
				Iterator<TagNode> i = nodes2.iterator();
				while (i.hasNext()) {
					initBook(cleaner, book, i);
				}
				
				book.setDetailedLink(url);
				book.setSource("中华古籍善本国际联合书目系统");
				
				XPather xPather = new XPather("//a[//span[@class=\"dhzt\"]]");
				Object[] ns = xPather.evaluateAgainstNode(tagNode);
				
				for (Object o : ns) {
					String text = replaceCharacter(((TagNode) o).getText().toString());
					if(text.indexOf("下载图片") == 0){
						String link = ((TagNode) o).getAttributeByName("href");
						ImmutableList<String> of = ImmutableList.of("http://res4.nlc.gov.cn" + link);
						book.setBookImages(of);
					}
				}

				
				this.mongoTemplate.save(book);
				System.out.println(url);
				succs.add(url);
				
			} catch (MalformedURLException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			} catch (XPatherException e) {
				e.printStackTrace();
			} catch (Exception e) {
				e.printStackTrace();
			}
			
			j = j + 1;
			System.out.println("保存到了第" + j + "条");
		}
		this.urls.removeAll(succs);
		if(!this.urls.isEmpty())
			catchRemain();
	}

	private String getNextText(Iterator<TagNode> i) {
		return replaceCharacter(i.next().getText().toString());
	}
	
	
	@Test
	public void saveBookFromHtml() {
		CleanerProperties props = new CleanerProperties();
		 
		// set some properties to non-default values
		props.setTranslateSpecialEntities(true);
		props.setTransResCharsToNCR(true);
		props.setOmitComments(true);
		
		HtmlCleaner cleaner = new HtmlCleaner(props);
		
		TagNode tagNode = null;
		int i = 1;
		for (String url : this.allUrls) {
			try {
				AncientBook book = new AncientBook();
				URL getUrl = new URL(url);
				HttpURLConnection connection = (HttpURLConnection) getUrl.openConnection();
				connection.setConnectTimeout(7000);
				connection.setReadTimeout(7000);
				connection.connect();
				/**
				 * 设置编码,否则中文乱码
				 */
				BufferedReader reader = new BufferedReader(
						new InputStreamReader(connection.getInputStream(), "utf-8"));
				
				tagNode = cleaner.clean(reader);
		
				// 著录编号
				List<String> indexNum = getRegInnerHtml(cleaner, tagNode, reg_1);
				book.setIndexNumber(replaceCharacter(indexNum.get(0)));

				// 正题名,其他题名,责任者,版本类型
				// 正题名,责任者,版本类型,版本信息
				List<String> booknames = getRegInnerHtml(cleaner, tagNode, reg_2);
				book.setBookName(replaceCharacter(booknames.get(0)));
				// 版本信息,载体形态
				// 载体形态
				List<String> carrierShapes = getRegInnerHtml(cleaner, tagNode, reg_3);

				if (isHaveOtherName(tagNode)) {
					book.setOtherBookNames(Arrays.asList(replaceCharacter(booknames.get(1)).split("<br/>")));
					book.setAuthors(Arrays.asList(replaceCharacter(booknames.get(2)).split(";")));
					book.setVersion(replaceCharacter(booknames.get(3)) + replaceCharacter(carrierShapes.get(0)));
					book.setCarrierShape(carrierShapes.get(1).trim());
				} else {
					book.setAuthors(Arrays.asList(replaceCharacter(booknames.get(1)).split(";")));
					book.setVersion(replaceCharacter(booknames.get(2)) + replaceCharacter(booknames.get(3)));
					book.setCarrierShape(replaceCharacter(carrierShapes.get(0)));
				}

				// 行款版式,附注,四部分类,收藏单位
				List<String> frames = getRegInnerHtml(cleaner, tagNode, reg_4);

				book.setFrame(frames.size() >= 1 ? replaceCharacter(frames.get(0)) : "");
				book.setAppendix(frames.size() >= 2 ? replaceCharacter(frames.get(1)) : "");
				book.setCategory(frames.size() >= 3 ? replaceCharacter(frames.get(2)) : "");
				book.setSource(frames.size() >= 4 ? replaceCharacter(frames.get(3)) : "");
				
				book.setDetailedLink(url);
				
				XPather xPather = new XPather("//a[//span[@class=\"dhzt\"]]");
				Object[] ns = xPather.evaluateAgainstNode(tagNode);
				
				for (Object o : ns) {
					String text = replaceCharacter(((TagNode) o).getText().toString());
					if(text.indexOf("下载图片") == 0){
						String link = ((TagNode) o).getAttributeByName("href");
						//System.out.println(link);
						ImmutableList<String> of = ImmutableList.of("http://res4.nlc.gov.cn" + link);
						book.setBookImages(of);
					}
				}
				
				
				
				
				// 典藏号
				getRegInnerHtml(cleaner, tagNode, reg_5);

				this.mongoTemplate.save(book);
				System.out.println(url);
				
			} catch (MalformedURLException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			} catch (XPatherException e) {
				e.printStackTrace();
			} catch (Exception e) {
				e.printStackTrace();
			}
			
			i = i + 1;
			System.out.println("保存到了第" + i + "条");
		}
	}

	private List<String> getRegInnerHtml(HtmlCleaner cleaner, TagNode tagNode,
			String reg_1) throws XPatherException {
		Object[] ns = tagNode.evaluateXPath(getXpathExpression(reg_1));
		List<String> contents = Lists.newArrayList();
		for(Object o : ns){
			String inner = cleaner.getInnerHtml((TagNode) o); 
			System.out.println(inner);
			contents.add(inner);
		}
		return contents;
	}

	private void getURL() {
		initAllUrls(this.Lines1000);
		initAllUrls(this.Lines2000);
		initAllUrls(this.Lines3000);
		initAllUrls(this.Lines4000);
	}

	private void initAllUrls(File file){
		HtmlCleaner cleaner = new HtmlCleaner();
		try {
			TagNode node = cleaner.clean(file);
			String xPathExpression = "//td[@class=\"ljzt3\"]/a";
			Object[] ns = node.evaluateXPath(xPathExpression);
			for(Object o : ns){
				TagNode n = (TagNode) o;
				//String u = "http://res4.nlc.gov.cn" + n.getAttributeByName("href");
				String u = n.getAttributeByName("href");
				this.allUrls.add(u);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}catch (XPatherException e) {
			e.printStackTrace();
		}
	}
	
	private boolean isHaveOtherName(TagNode tagNode){
		String reg = "border-bottom:#e8e3d3 solid 1px; border-left:#d6d6d6 solid 1px;";
		HtmlCleaner cleaner = new HtmlCleaner();
		
		try {
			Object[] ns = tagNode.evaluateXPath(getXpathExpression(reg));
			for(Object o : ns){
				String inner = cleaner.getInnerHtml((TagNode) o); 
				if(inner.contains("其他题名")) return true;
			}
		} catch (XPatherException e) {
			e.printStackTrace();
		}
		return false;
	}
}
